# Probe for testing CVMFS functioning at WLCG grid sites, the probe will test the basic functionality of the
# cvmfs client and tests per mandatory and optional repository. In case of nfs exports a test will be done
# whether the mount point is of type nfs, no further tests can be done in this case

# Author: stefan.roiser@cern.ch

# History
# 1.0 : initial version
# 1.1 : testing of optional repositories, testing of VO_{VO}_SW_DIR variable points to CVMFS mount point
# 1.2 : testing of nfs export on sites

## variables to configure the probe
DEF_REPOSITORIES=$1        # space delimited list of repositories to test and must work
                                       # any test failing for those will not update the return code of the probe, 
                                       # the error/warning will still be printed
##

# 
# some globally used variables
#
SCRIPT_REVISION="1.2c"

STATUS_OK=0
STATUS_WARNING=1     # Check timed out or CernVM-FS resource consumption high or
                     # previous error status detected
STATUS_CRITICAL=2    # CernVM-FS not working
STATUS_UNKNOWN=3     # Internal or usage error

INFO=0          
WARNING=1     # CernVM-FS resource consumption high or 
ERROR=2    # CernVM-FS not working

BRIEF_INFO="OK: cvmfs checks passed"
LONG_INFO=""
RETURN_STATUS=$INFO



#
# helper functions
# 

final_report() {
  if [ $1 -eq $ERROR ] ; then /bin/echo $LONG_INFO; fi
  if [ $1 -eq $INFO ] ; then /bin/echo "$DEF_REPOSITORIES is ok "; fi
  exit $1
}

report() {
  #return;
  local kind; kind=$1
  local msg; msg=$2
  local strict; strict=$3
  local stat; stat=${!kind}
  LONG_INFO="${LONG_INFO}${kind}: ${msg}\n"
  if [ $stat -gt $RETURN_STATUS -a "x$strict" = "xTrue" ] ; 
  then 
    RETURN_STATUS=$stat
  fi 

  if [ $stat -eq $ERROR ] ; then final_report $stat; fi
}


get_xattr() {
  local _resultvar; _resultvar=$1
  local xattrname; xattrname=$2
  local repo; repo=$3
  local strict; strict=$4
  local result; result=''
     
  result=`/usr/bin/attr -q -g $xattrname $repo`
  if [ $? -ne 0 ]; then
    report "WARNING: failed to read $xattrname attribute in $repo" $strict
  fi

  eval $_resultvar="'$result'"
}



test_basic_functionality() {

  # Read repository config
  if [ -f /etc/cvmfs/config.sh ]
  then
    report "INFO" "Executing /etc/cvmfs/config.sh" "True"
    . /etc/cvmfs/config.sh
  else report "ERROR" "cannot continue /etc/cvmfs/config.sh missing" "True"
  fi

  # Read configuration
  cvmfs_readconfig
  if [ $? -ne 0 ]; then report "ERROR" "failed to read CernVM-FS configuration" "True" ; fi 

}


test_repository() {
  local repo; repo=$1
  local strict; strict=$2


  FQRN=`cvmfs_mkfqrn $repo`
  ORG=`cvmfs_getorg $FQRN`
  local mnt_repo=${CVMFS_MOUNT_DIR}/${FQRN}
  #report "INFO" "- Testing repository $mnt_repo -" $strict
  cvmfs_readconfig $FQRN
  if [ $? -ne 0 ] ; then report "ERROR" "failed to read $FQRN configuration" $strict
  else report "INFO" "configuration successfully read" $strict
  fi

  cd "${mnt_repo}" && ls . > /dev/null
  
  if [ $? -ne 0 ];   then report "ERROR" "failed to access $FQRN" $strict
  else report "INFO" "repository $mnt_repo available" $strict
  fi

  get_xattr VERS_LOADED version $mnt_repo $strict
  #report "INFO" "CVMFS version loaded $VERS_LOADED" $strict

  VERS_INSTALLED=`/usr/bin/cvmfs2 --version 2>&1 | awk '{ print $NF }'`
  #report "INFO" "CVMFS version installed $VERS_INSTALLED" $strict

  get_xattr REPO_REVISION revision $mnt_repo $strict
  #report "INFO" "repository revision $REPO_REVISION" $strict

  get_xattr CVMFS_IOERR nioerr $mnt_repo $strict
  if [ $CVMFS_IOERR -gt 0 ] ; then  /usr/bin/cvmfs_config reload $repo; report "ERROR" "$CVMFS_IOERR I/O errors detected" $strict; fi

  get_xattr CVMFS_USEDFD usedfd $mnt_repo $strict
  get_xattr CVMFS_MAXFD maxfd $mnt_repo $strict
  CVMFS_FDRATIO=$[$CVMFS_USEDFD*100/$CVMFS_MAXFD]
  if [ $CVMFS_FDRATIO -gt 95 ]; then report "ERROR" "file descriptors almost exhausted (${CVMFS_FDRATIO}%)" $strict; fi  
  if [ $CVMFS_FDRATIO -gt 80 ]; then report "WARNING" "low on open file descriptors (${CVMFS_FDRATIO}%)" $strict; fi

  get_xattr CVMFS_NUMCTLG nclg $mnt_repo $strict
  if [ $[$CVMFS_NUMCTLG*10] -gt $CVMFS_MAXFD ]; then report "WARNING: high number of catalogs loaded ($CVMFS_NUMCTLG)" $strict; fi

  get_xattr CVMFS_PID pid $mnt_repo $strict
  CVMFS_MEM=`/bin/ps -p $CVMFS_PID -o rss= | /bin/sed 's/ //g'`
  TOT_MEM=`cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'`
  if [ $PIPESTATUS -ne 0 ] 
  then report "WARNING" "failed to read memory consumption" $strict
  else report "INFO" "memory consumption $CVMFS_MEM kB (machine has $TOT_MEM kB)" $strict
  fi

  if [ -f "${mnt_repo}/.cvmfsdirtab" ]; then
    cat "${mnt_repo}/.cvmfsdirtab" > /dev/null 2>&1
    if [ $? -ne 0 ]; then report "ERROR" "failed to read .cvmfsdirtab from repository" $strict; fi
  fi


  # connectivity tests (just info now: no longer report these as warnings
  CVMFS_HOSTS=`/bin/echo "$CVMFS_SERVER_URL" | /bin/sed 's/,\|;/ /g' | sed s/@org@/$ORG/g | sed s/@fqrn@/$FQRN/g`
  CVMFS_PROXIES=`/bin/echo "$CVMFS_HTTP_PROXY" | /bin/sed 's/;\||/ /g'`
  #report "INFO" "testing network connectivity - SERVERS cross PROXIES" $strict
  #report "INFO" "Servers: $CVMFS_HOSTS" $strict
  #report "INFO" "Proxies: $CVMFS_PROXIES" $strict
  get_xattr CVMFS_TIMEOUT_PROXY timeout $mnt_repo $strict
  get_xattr CVMFS_TIMEOUT_DIRECT timeout_direct $mnt_repo $strict
  #for HOST in $CVMFS_HOSTS
  #do
  #  for PROXY in $CVMFS_PROXIES
  #  do
  #    if [ $PROXY != "DIRECT" ]; then
  #      PROXY_ENV="env http_proxy=$PROXY"
  #      TIMEOUT=$CVMFS_TIMEOUT_PROXY
  #    else
  #      PROXY_ENV=
  #      TIMEOUT=$CVMFS_TIMEOUT_DIRECT
  #    fi
  #    URL="${HOST}/.cvmfspublished"
  #    $PROXY_ENV /usr/bin/curl -H "Pragma:" -f --connect-timeout $TIMEOUT $URL > /dev/null 2>&1
  #    ret=$?
  #    if [ $ret -ne 0 ]; then report "INFO" "connection problem ($HOST via $PROXY) (curl return code: $ret)" $strict ; fi
  #  done
  #done
  #report "INFO" "end of network connectivity tests" $strict
}


#
# Start of test procedure
#

if [ "$VO_LHCB_SW_DIR" = "" ] ; then
  VO_LHCB_SW_DIR="/cvmfs/lhcb.cern.ch"
  #report "INFO" "No VO_LHCB_SW_DIR set. Using $VO_LHCB_SW_DIR for testing" "True"
fi

#report "INFO" "-- Executing basic tests --" "True"
test_basic_functionality

#report "INFO" "-- Testing repositories --" "True"
for REPO in $DEF_REPOSITORIES ; do test_repository $REPO True ; done



# 
# Report
#
final_report $INFO

